{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Novel Simulations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Define and train a VGG Convolutional Neural Network (CNN) using CIFAR-10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import memtorch\n",
    "from memtorch.utils import LoadCIFAR10\n",
    "\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self, inflation_ratio=1):\n",
    "        super(Net, self).__init__()\n",
    "        self.conv0 = nn.Conv2d(in_channels=3, out_channels=128*inflation_ratio, kernel_size=3, stride=1, padding=1)\n",
    "        self.bn0 = nn.BatchNorm2d(num_features=128*inflation_ratio)\n",
    "        self.act0 = nn.ReLU()\n",
    "        self.conv1 = nn.Conv2d(in_channels=128*inflation_ratio, out_channels=128*inflation_ratio, kernel_size=3, padding=1)\n",
    "        self.bn1 = nn.BatchNorm2d(num_features=128*inflation_ratio)\n",
    "        self.act1 = nn.ReLU()\n",
    "        self.conv2 = nn.Conv2d(in_channels=128*inflation_ratio, out_channels=256*inflation_ratio, kernel_size=3, padding=1)\n",
    "        self.bn2 = nn.BatchNorm2d(num_features=256*inflation_ratio)\n",
    "        self.act2 = nn.ReLU()\n",
    "        self.conv3 = nn.Conv2d(in_channels=256*inflation_ratio, out_channels=256*inflation_ratio, kernel_size=3, padding=1)\n",
    "        self.bn3 = nn.BatchNorm2d(num_features=256*inflation_ratio)\n",
    "        self.act3 = nn.ReLU()\n",
    "        self.conv4 = nn.Conv2d(in_channels=256*inflation_ratio, out_channels=512*inflation_ratio, kernel_size=3, padding=1)\n",
    "        self.bn4 = nn.BatchNorm2d(num_features=512*inflation_ratio)\n",
    "        self.act4 = nn.ReLU()\n",
    "        self.conv5 = nn.Conv2d(in_channels=512*inflation_ratio, out_channels=512, kernel_size=3, padding=1)\n",
    "        self.bn5 = nn.BatchNorm2d(num_features=512)\n",
    "        self.act5 = nn.ReLU()\n",
    "        self.fc6 = nn.Linear(in_features=512*4*4, out_features=1024)\n",
    "        self.bn6 = nn.BatchNorm1d(num_features=1024)\n",
    "        self.act6 = nn.ReLU()\n",
    "        self.fc7 = nn.Linear(in_features=1024, out_features=1024)\n",
    "        self.bn7 = nn.BatchNorm1d(num_features=1024)\n",
    "        self.act7 = nn.ReLU()\n",
    "        self.fc8 = nn.Linear(in_features=1024, out_features=10)\n",
    "\n",
    "    def forward(self, input):\n",
    "        x = self.act0(self.bn0(self.conv0(input)))\n",
    "        x = self.act1(self.bn1(F.max_pool2d(self.conv1(x), 2)))\n",
    "        x = self.act2(self.bn2(self.conv2(x)))\n",
    "        x = self.act3(self.bn3(F.max_pool2d(self.conv3(x), 2)))\n",
    "        x = self.act4(self.bn4(self.conv4(x)))\n",
    "        x = self.act5(self.bn5(F.max_pool2d(self.conv5(x), 2)))\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.act6(self.bn6(self.fc6(x)))\n",
    "        x = self.act7(self.bn7(self.fc7(x)))\n",
    "        return self.fc8(x)\n",
    "    \n",
    "    \n",
    "def test(model, test_loader):\n",
    "    correct = 0\n",
    "    for batch_idx, (data, target) in enumerate(test_loader):        \n",
    "        output = model(data.to(device))\n",
    "        pred = output.data.max(1)[1]\n",
    "        correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\n",
    "\n",
    "    return 100. * float(correct) / float(len(test_loader.dataset))\n",
    "\n",
    "device = torch.device('cpu' if 'cpu' in memtorch.__version__ else 'cuda')\n",
    "epochs = 50\n",
    "train_loader, validation_loader, test_loader = LoadCIFAR10(batch_size=256, validation=False)\n",
    "model = Net().to(device)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "learning_rate = 1e-2\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "best_accuracy = 0\n",
    "for epoch in range(0, epochs):\n",
    "    print('Epoch: [%d]\\t\\t' % (epoch + 1), end='')\n",
    "    if epoch % 20 == 0:\n",
    "        learning_rate = learning_rate * 0.1\n",
    "        for param_group in optimizer.param_groups:\n",
    "            param_group['lr'] = learning_rate\n",
    "\n",
    "    model.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader):\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data.to(device))\n",
    "        loss = criterion(output, target.to(device))\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    accuracy = test(model, test_loader)\n",
    "    print('%2.2f%%' % accuracy)\n",
    "    if accuracy > best_accuracy:\n",
    "        torch.save(model.state_dict(), 'trained_model.pt')\n",
    "        best_accuracy = accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Load and test the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "model = Net().to(device)\n",
    "try:\n",
    "    model.load_state_dict(torch.load('trained_model.pt'), strict=False)\n",
    "except:\n",
    "    raise Exception('trained_model.pt has not been found.')\n",
    "    \n",
    "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Import seaborn and define an appropriate color-palette"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "\n",
    "\n",
    "palette = [\"#DA4453\", \"#8CC152\", \"#4A89DC\", \"#F6BB42\", \"#B600B0\", \"#535353\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Device-device variability investigation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from memtorch.mn.Module import patch_model\n",
    "from memtorch.map.Parameter import naive_map\n",
    "from memtorch.bh.crossbar.Program import naive_program\n",
    "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n",
    "\n",
    "\n",
    "def trial(r_on, r_off, sigma):\n",
    "    model_ = copy.deepcopy(model)\n",
    "    reference_memristor = memtorch.bh.memristor.VTEAM\n",
    "    reference_memristor_params = {'time_series_resolution': 1e-10,\n",
    "                                  'r_off': memtorch.bh.StochasticParameter(loc=r_off, scale=sigma*2, min=1),\n",
    "                                  'r_on': memtorch.bh.StochasticParameter(loc=r_on, scale=sigma, min=1)}\n",
    "\n",
    "    patched_model = patch_model(copy.deepcopy(model_),\n",
    "                              memristor_model=reference_memristor,\n",
    "                              memristor_model_params=reference_memristor_params,\n",
    "                              module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n",
    "                              mapping_routine=naive_map,\n",
    "                              transistor=True,\n",
    "                              programming_routine=None)\n",
    "    \n",
    "    patched_model.tune_()\n",
    "    return test(patched_model, test_loader)\n",
    "\n",
    "df = pd.DataFrame(columns=['sigma', 'test_set_accuracy'])\n",
    "r_on = 200\n",
    "r_off = 500\n",
    "sigma_values = np.linspace(0, 100, 21)\n",
    "for sigma in sigma_values:\n",
    "    df = df.append({'sigma': sigma, 'test_set_accuracy': trial(r_on, r_off, sigma)}, ignore_index=True)\n",
    "    \n",
    "df.to_csv('variability.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('variability.csv')\n",
    "f = plt.figure()\n",
    "plt.axhline(y=10, color='k', linestyle='--', zorder=1)\n",
    "b = plt.bar(df['sigma'], df['test_set_accuracy'], width=2.5, zorder=2)\n",
    "plt.xlabel('$\\sigma$')\n",
    "plt.ylabel('CIFAR-10 Test-set Accuracy (%)')\n",
    "for bar in b:\n",
    "    bar.set_edgecolor('black')\n",
    "    bar.set_facecolor(palette[0])\n",
    "    bar.set_linewidth(1)\n",
    "    \n",
    "f.tight_layout()\n",
    "plt.grid()\n",
    "plt.savefig(\"P1.svg\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Finite conductance states investigation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from memtorch.mn.Module import patch_model\n",
    "from memtorch.map.Parameter import naive_map\n",
    "from memtorch.bh.crossbar.Program import naive_program\n",
    "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n",
    "\n",
    "\n",
    "def trial(r_on, r_off, finite_states):\n",
    "    model_ = copy.deepcopy(model)\n",
    "    reference_memristor = memtorch.bh.memristor.VTEAM\n",
    "    reference_memristor_params = {'time_series_resolution': 1e-10,\n",
    "                                  'r_off': r_off,\n",
    "                                  'r_on': r_on}\n",
    "\n",
    "    patched_model = patch_model(copy.deepcopy(model_),\n",
    "                              memristor_model=reference_memristor,\n",
    "                              memristor_model_params=reference_memristor_params,\n",
    "                              module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n",
    "                              mapping_routine=naive_map,\n",
    "                              transistor=True,\n",
    "                              programming_routine=None)\n",
    "    \n",
    "    patched_model = apply_nonidealities(patched_model,\n",
    "                            non_idealities=[memtorch.bh.nonideality.NonIdeality.FiniteConductanceStates],\n",
    "                            conductance_states = int(finite_states))\n",
    "    \n",
    "    patched_model.tune_()\n",
    "    return test(patched_model, test_loader)\n",
    "\n",
    "df = pd.DataFrame(columns=['finite_states', 'test_set_accuracy'])\n",
    "r_on = 200\n",
    "r_off = 500\n",
    "finite_state_values = np.linspace(1, 10, 10)\n",
    "for finite_states in finite_state_values:\n",
    "    df = df.append({'finite_states': finite_states, 'test_set_accuracy': trial(r_on, r_off, finite_states)}, ignore_index=True)\n",
    "    \n",
    "df.to_csv('finite_states.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('finite_states.csv')\n",
    "f = plt.figure()\n",
    "plt.axhline(y=10, color='k', linestyle='--', zorder=1)\n",
    "b = plt.bar(df['finite_states'], df['test_set_accuracy'], width=0.5, zorder=2)\n",
    "plt.xlabel('Number of Finite States')\n",
    "plt.ylabel('CIFAR-10 Test-set Accuracy (%)')\n",
    "plt.xticks(df['finite_states'])\n",
    "for bar in b:\n",
    "    bar.set_edgecolor('black')\n",
    "    bar.set_facecolor(palette[0])\n",
    "    bar.set_linewidth(1)\n",
    "    \n",
    "f.tight_layout()\n",
    "plt.grid()\n",
    "plt.savefig(\"P2.svg\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Device failure investigation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from memtorch.mn.Module import patch_model\n",
    "from memtorch.map.Parameter import naive_map\n",
    "from memtorch.bh.crossbar.Program import naive_program\n",
    "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n",
    "\n",
    "\n",
    "def trial(r_on, r_off, lrs_proportion, hrs_proportion):\n",
    "    model_ = copy.deepcopy(model)\n",
    "    reference_memristor = memtorch.bh.memristor.VTEAM\n",
    "    reference_memristor_params = {'time_series_resolution': 1e-10,\n",
    "                                  'r_off': r_off,\n",
    "                                  'r_on': r_on}\n",
    "\n",
    "    patched_model = patch_model(copy.deepcopy(model_),\n",
    "                              memristor_model=reference_memristor,\n",
    "                              memristor_model_params=reference_memristor_params,\n",
    "                              module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n",
    "                              mapping_routine=naive_map,\n",
    "                              transistor=True,\n",
    "                              programming_routine=None)\n",
    "    \n",
    "    patched_model = apply_nonidealities(patched_model,\n",
    "                            non_idealities=[memtorch.bh.nonideality.NonIdeality.DeviceFaults],\n",
    "                            lrs_proportion=lrs_proportion,\n",
    "                            hrs_proportion=hrs_proportion,\n",
    "                            electroform_proportion=0)\n",
    "    \n",
    "    patched_model.tune_()\n",
    "    return test(patched_model, test_loader)\n",
    "\n",
    "df_lrs_hrs = pd.DataFrame(columns=['failure_percentage', 'test_set_accuracy'])\n",
    "df_lrs = pd.DataFrame(columns=['failure_percentage', 'test_set_accuracy'])\n",
    "df_hrs = pd.DataFrame(columns=['failure_percentage', 'test_set_accuracy'])\n",
    "r_on = 200\n",
    "r_off = 500\n",
    "failures = np.linspace(0, 0.25, 11)\n",
    "\n",
    "for failure in failures:\n",
    "    df_lrs_hrs = df_lrs_hrs.append({'failure_percentage': failure, 'test_set_accuracy': trial(r_on, r_off, failure, failure)}, ignore_index=True)\n",
    "    df_lrs = df_lrs.append({'failure_percentage': failure, 'test_set_accuracy': trial(r_on, r_off, failure, 0)}, ignore_index=True)    \n",
    "    df_hrs = df_hrs.append({'failure_percentage': failure, 'test_set_accuracy': trial(r_on, r_off, 0, failure)}, ignore_index=True)  \n",
    "    \n",
    "df_lrs_hrs.to_csv('failure_lrs_hrs.csv', index=False)\n",
    "df_lrs.to_csv('failure_lrs.csv', index=False)\n",
    "df_hrs.to_csv('failure_hrs.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "\n",
    "\n",
    "df_lrs_hrs = pd.read_csv('failure_lrs_hrs.csv')\n",
    "df_lrs = pd.read_csv('failure_lrs.csv')\n",
    "df_hrs = pd.read_csv('failure_hrs.csv')\n",
    "f = plt.figure()\n",
    "plt.axhline(y=10, color='k', linestyle='--', zorder=1)\n",
    "concat = pd.concat([df_lrs_hrs['failure_percentage'], \n",
    "                  df_lrs_hrs['test_set_accuracy'], \n",
    "                  df_lrs['test_set_accuracy'], \n",
    "                  df_hrs['test_set_accuracy']], \n",
    "                  axis=1)\n",
    "concat.columns = ['failure_percentage', 'lrs_hrs', 'lrs', 'hrs']\n",
    "data = pd.DataFrame(columns=['failure_percentage', 'state', 'test_set_accuracy'])\n",
    "for index, row in concat.iterrows():\n",
    "    data = data.append({'failure_percentage': row['failure_percentage'] * 100, 'state': 'lrs_hrs', 'test_set_accuracy': row['lrs_hrs']}, ignore_index=True)\n",
    "    data = data.append({'failure_percentage': row['failure_percentage'] * 100, 'state': 'lrs', 'test_set_accuracy': row['lrs']}, ignore_index=True)\n",
    "    data = data.append({'failure_percentage': row['failure_percentage'] * 100, 'state': 'hrs', 'test_set_accuracy': row['hrs']}, ignore_index=True)\n",
    "    \n",
    "data['state'] = data['state'].map({'lrs_hrs': '$R_{ON}$ and $R_{OFF}$', 'lrs': '$R_{ON}$', 'hrs': '$R_{OFF}$'})\n",
    "h = sns.barplot(x=\"failure_percentage\", hue=\"state\", y=\"test_set_accuracy\", data=data, zorder=2, edgecolor='black', linewidth='1', palette=sns.color_palette(palette), saturation=1)\n",
    "plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%1.1f'))\n",
    "plt.xticks(np.arange(11), np.arange(0, 25 + 2.5, step=2.5))\n",
    "h.legend(loc=1)\n",
    "plt.xlabel('Device Failure (%)')\n",
    "plt.ylabel('CIFAR-10 Test-set Accuracy (%)') \n",
    "f.tight_layout()\n",
    "plt.grid()\n",
    "plt.savefig(\"P3.svg\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. First novel simulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from memtorch.mn.Module import patch_model\n",
    "from memtorch.map.Parameter import naive_map\n",
    "from memtorch.bh.crossbar.Program import naive_program\n",
    "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n",
    "\n",
    "\n",
    "def trial(num_conductance_states, g_ratio, sigma):\n",
    "    model_ = copy.deepcopy(model)\n",
    "    r_on = 200\n",
    "    reference_memristor = memtorch.bh.memristor.VTEAM\n",
    "    reference_memristor_params = {'time_series_resolution': 1e-10,\n",
    "                                  'r_off': memtorch.bh.StochasticParameter(loc=r_on*g_ratio, scale=sigma*2, min=1),\n",
    "                                  'r_on': memtorch.bh.StochasticParameter(loc=r_on, scale=sigma, min=1)}\n",
    "    patched_model = patch_model(copy.deepcopy(model_),\n",
    "                              memristor_model=reference_memristor,\n",
    "                              memristor_model_params=reference_memristor_params,\n",
    "                              module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n",
    "                              mapping_routine=naive_map,\n",
    "                              transistor=True,\n",
    "                              programming_routine=None)\n",
    "    patched_model = apply_nonidealities(patched_model,\n",
    "                            non_idealities=[memtorch.bh.nonideality.NonIdeality.FiniteConductanceStates],\n",
    "                            conductance_states=int(num_conductance_states))\n",
    "    patched_model.tune_()\n",
    "    return test(patched_model, test_loader)\n",
    "\n",
    "std_devs = [0, 20, 100]\n",
    "g_ratios = [2 ** n for n in range(6)]\n",
    "conductance_states = np.linspace(2, 10, 9)\n",
    "for std_dev in std_devs:\n",
    "    df = pd.DataFrame(columns=['conductance_states', 'g_ratio', 'test_set_accuracy'])\n",
    "    for g_ratio in g_ratios:\n",
    "        for num_conductance_states in conductance_states:\n",
    "            test_set_accuracy = trial(num_conductance_states, g_ratio, std_dev)\n",
    "            df = df.append({'conductance_states': num_conductance_states, \n",
    "                            'g_ratio': g_ratio, \n",
    "                            'test_set_accuracy': test_set_accuracy}, ignore_index=True)\n",
    "    \n",
    "    df.to_csv('S1_std_dev_%d.csv' % std_dev, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "\n",
    "f = plt.figure(figsize=(16, 4))\n",
    "plt.subplot(1, len(std_devs), 1)\n",
    "for plot_index, std_dev in enumerate(std_devs):\n",
    "    plt.subplot(1, len(std_devs), plot_index + 1)\n",
    "    plt.axhline(y=10, color='k', linestyle='--', zorder=1)\n",
    "    data = pd.read_csv('S1_std_dev_%d.csv' % std_dev)\n",
    "    h = sns.barplot(x=\"conductance_states\", hue=\"g_ratio\", y=\"test_set_accuracy\", data=data, zorder=2, edgecolor='black', linewidth='1', palette=sns.color_palette(palette), saturation=1)\n",
    "    plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%1.0f'))\n",
    "    plt.xticks(np.arange(0, len(conductance_states)), map(lambda n: \"%d\" % n, conductance_states))\n",
    "    leg = h.axes.get_legend()\n",
    "    leg.set_title('RON/ROFF Ratio')\n",
    "    h.legend(loc=1)\n",
    "    plt.title('$\\sigma$ = %d' % std_dev)\n",
    "    if plot_index == 0:\n",
    "        plt.xlabel('Number of Finite States')\n",
    "        plt.ylabel('CIFAR-10 Test-set Accuracy (%)')\n",
    "    else:\n",
    "        plt.xlabel('')\n",
    "        plt.ylabel('')\n",
    "        \n",
    "    plt.grid()\n",
    "     \n",
    "f.tight_layout()\n",
    "plt.savefig(\"S1.svg\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Second novel simulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from memtorch.mn.Module import patch_model\n",
    "from memtorch.map.Parameter import naive_map\n",
    "from memtorch.bh.crossbar.Program import naive_program\n",
    "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n",
    "\n",
    "\n",
    "def trial(num_conductance_states, lrs_failure_rate, hrs_failure_rate, sigma):\n",
    "    model_ = copy.deepcopy(model)\n",
    "    r_on = 200\n",
    "    r_off = 500\n",
    "    reference_memristor = memtorch.bh.memristor.VTEAM\n",
    "    reference_memristor_params = {'time_series_resolution': 1e-10,\n",
    "                                  'r_off': memtorch.bh.StochasticParameter(loc=r_on*g_ratio, scale=sigma*2, min=1),\n",
    "                                  'r_on': memtorch.bh.StochasticParameter(loc=r_off, scale=sigma, min=1)}\n",
    "    patched_model = patch_model(copy.deepcopy(model_),\n",
    "                              memristor_model=reference_memristor,\n",
    "                              memristor_model_params=reference_memristor_params,\n",
    "                              module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n",
    "                              mapping_routine=naive_map,\n",
    "                              transistor=True,\n",
    "                              programming_routine=None)\n",
    "    patched_model = apply_nonidealities(patched_model,\n",
    "                            non_idealities=[memtorch.bh.nonideality.NonIdeality.FiniteConductanceStates,\n",
    "                                            memtorch.bh.nonideality.NonIdeality.DeviceFaults],\n",
    "                            conductance_states=int(num_conductance_states),\n",
    "                            lrs_proportion=lrs_failure_rate,\n",
    "                            hrs_proportion=hrs_failure_rate,\n",
    "                            electroform_proportion=0)\n",
    "    patched_model.tune_()\n",
    "    return test(patched_model, test_loader)\n",
    "\n",
    "std_devs = [0, 20, 100]\n",
    "failure_rates = np.linspace(0, 0.25, 6)\n",
    "conductance_states = np.linspace(2, 10, 9)\n",
    "\n",
    "# LRS\n",
    "for std_dev in std_devs:\n",
    "    df = pd.DataFrame(columns=['conductance_states', 'failure_rate', 'test_set_accuracy'])\n",
    "    for failure_rate in failure_rates:\n",
    "        for num_conductance_states in conductance_states:\n",
    "            test_set_accuracy = trial(num_conductance_states, failure_rate, 0, std_dev)\n",
    "            df = df.append({'conductance_states': num_conductance_states, \n",
    "                            'failure_rate': failure_rate, \n",
    "                            'test_set_accuracy': test_set_accuracy}, ignore_index=True)\n",
    "    \n",
    "    df.to_csv('S2_LRS_std_dev_%d.csv' % std_dev, index=False)\n",
    "    \n",
    "# HRS\n",
    "for std_dev in std_devs:\n",
    "    df = pd.DataFrame(columns=['conductance_states', 'failure_rate', 'test_set_accuracy'])\n",
    "    for failure_rate in failure_rates:\n",
    "        for num_conductance_states in conductance_states:\n",
    "            test_set_accuracy = trial(num_conductance_states, 0, failure_rate, std_dev)\n",
    "            df = df.append({'conductance_states': num_conductance_states, \n",
    "                            'failure_rate': failure_rate, \n",
    "                            'test_set_accuracy': test_set_accuracy}, ignore_index=True)\n",
    "    \n",
    "    df.to_csv('S2_HRS_std_dev_%d.csv' % std_dev, index=False)\n",
    "    \n",
    "# LRS and HRS\n",
    "for std_dev in std_devs:\n",
    "    df = pd.DataFrame(columns=['conductance_states', 'failure_rate', 'test_set_accuracy'])\n",
    "    for failure_rate in failure_rates:\n",
    "        for num_conductance_states in conductance_states:\n",
    "            test_set_accuracy = trial(num_conductance_states, failure_rate, failure_rate, std_dev)\n",
    "            df = df.append({'conductance_states': num_conductance_states, \n",
    "                            'failure_rate': failure_rate, \n",
    "                            'test_set_accuracy': test_set_accuracy}, ignore_index=True)\n",
    "    \n",
    "    df.to_csv('S2_LRS_HRS_std_dev_%d.csv' % std_dev, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "from matplotlib.ticker import FormatStrFormatter\n",
    "\n",
    "f = plt.figure(figsize=(16, 12))\n",
    "plt.subplot(3, len(std_devs), 1)\n",
    "# LRS\n",
    "for plot_index, std_dev in enumerate(std_devs):\n",
    "    plt.subplot(3, len(std_devs), plot_index + 1)\n",
    "    plt.axhline(y=10, color='k', linestyle='--', zorder=1)\n",
    "    data = pd.read_csv('S2_LRS_std_dev_%d.csv' % std_dev)\n",
    "    h = sns.barplot(x=\"conductance_states\", hue=\"failure_rate\", y=\"test_set_accuracy\", data=data, zorder=2, edgecolor='black', linewidth='1', palette=sns.color_palette(palette), saturation=1)\n",
    "    plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%1.0f'))\n",
    "    plt.xticks(np.arange(0, len(conductance_states)), map(lambda n: \"%d\" % n, conductance_states))\n",
    "    leg = h.axes.get_legend()\n",
    "    leg.set_title('Device Failure (%)')\n",
    "    h.legend(loc=1)\n",
    "    plt.title('$\\sigma$ = %d' % std_dev)\n",
    "    if plot_index == 0:\n",
    "        plt.xlabel('Number of Finite States')\n",
    "        plt.ylabel('[LRS] CIFAR-10 Test-set Accuracy (%)')\n",
    "    else:\n",
    "        plt.xlabel('')\n",
    "        plt.ylabel('')\n",
    "        \n",
    "    plt.grid()\n",
    "        \n",
    "# HRS\n",
    "for plot_index, std_dev in enumerate(std_devs):\n",
    "    plt.subplot(3, len(std_devs), plot_index + 1 + len(std_devs))\n",
    "    plt.axhline(y=10, color='k', linestyle='--', zorder=1)\n",
    "    data = pd.read_csv('S2_HRS_std_dev_%d.csv' % std_dev)\n",
    "    h = sns.barplot(x=\"conductance_states\", hue=\"failure_rate\", y=\"test_set_accuracy\", data=data, zorder=2, edgecolor='black', linewidth='1', palette=sns.color_palette(palette), saturation=1)\n",
    "    plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%1.0f'))\n",
    "    plt.xticks(np.arange(0, len(conductance_states)), map(lambda n: \"%d\" % n, conductance_states))\n",
    "    leg = h.axes.get_legend()\n",
    "    leg.set_title('Device Failure (%)')\n",
    "    h.legend(loc=1)\n",
    "    plt.title('$\\sigma$ = %d' % std_dev)\n",
    "    if plot_index == 0:\n",
    "        plt.xlabel('Number of Finite States')\n",
    "        plt.ylabel('[HRS] CIFAR-10 Test-set Accuracy (%)')\n",
    "    else:\n",
    "        plt.xlabel('')\n",
    "        plt.ylabel('')\n",
    "        \n",
    "    plt.grid()\n",
    "        \n",
    "# LRS and HRS\n",
    "for plot_index, std_dev in enumerate(std_devs):\n",
    "    plt.subplot(3, len(std_devs), plot_index + 1 + 2 * len(std_devs))\n",
    "    plt.axhline(y=10, color='k', linestyle='--', zorder=1)\n",
    "    data = pd.read_csv('S2_LRS_HRS_std_dev_%d.csv' % std_dev)\n",
    "    h = sns.barplot(x=\"conductance_states\", hue=\"failure_rate\", y=\"test_set_accuracy\", data=data, zorder=2, edgecolor='black', linewidth='1', palette=sns.color_palette(palette), saturation=1)\n",
    "    plt.gca().xaxis.set_major_formatter(FormatStrFormatter('%1.0f'))\n",
    "    plt.xticks(np.arange(0, len(conductance_states)), map(lambda n: \"%d\" % n, conductance_states))\n",
    "    leg = h.axes.get_legend()\n",
    "    leg.set_title('Device Failure (%)')\n",
    "    h.legend(loc=1)\n",
    "    plt.title('$\\sigma$ = %d' % std_dev)\n",
    "    if plot_index == 0:\n",
    "        plt.xlabel('Number of Finite States')\n",
    "        plt.ylabel('[LRS and HRS] CIFAR-10 Test-set Accuracy (%)')\n",
    "    else:\n",
    "        plt.xlabel('')\n",
    "        plt.ylabel('')\n",
    "        \n",
    "    plt.grid()\n",
    "        \n",
    "f.tight_layout()\n",
    "plt.savefig(\"S2.svg\")\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
